'''
爱站平台
通过关键词，挖取相关词
Q群：170555357 
'''
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
import re,json,redis,time
from pskpackage.db import *


pool = redis.ConnectionPool(host='地址', port=6379, password='密码', db=数据表)

'''
id 数据表id，方便更新数据
keyword 关键词
redis_val  redis查询数据，查询异常重复查询
'''
def Dig_word(id,keywords,redis_val):
    global pool
    r = redis.Redis(connection_pool=pool)
    json_data = []
    return_data = {}

    url = "https://ci.aizhan.com/%s/" % (str(keywords))
    try:
        data = requests.get(url, timeout=5)
        soup = BeautifulSoup(data.text, "lxml")
    except:
        r.lpush('keyword_word', redis_val)  # 失败数据写入redis
        return False

    #获取关键词
    title = []
    for item in soup.find_all("td", attrs={'class': "title"}):
        if item.find("a"):
            title.append(item.find("a").text)
    return_data['title'] = title


    #获取流量
    flow = []
    for item in soup.find_all("td", attrs={'class': "center"}):
        if item.find("span"):
            flow.append(item.find("span").text)
    return_data['flow'] = flow


    # 获取收录
    level = []
    record = []
    for item in soup.find_all("td", attrs={'class':'level'}):
        if item.find("span"):
            span_class = item.find("span").find("span").get("class")
            span_class_val = span_class[0].replace('l', '')
            if int(span_class_val)==1:
                level.append("较小")
            elif int(span_class_val)==2:
                level.append("中等偏下")
            elif int(span_class_val) == 3:
                level.append("中等")
            elif int(span_class_val) == 4:
                level.append("中等偏上")
            elif int(span_class_val) == 5:
                level.append("高难")
        else:
            if item.text!="收录数" and item.text!="优化难度":
                record.append(item.text)

    return_data['level'] = level
    return_data['record'] = record

    json_data.append(return_data)
    json_data = json.dumps(json_data, ensure_ascii=False)

    update_key_word(id, json_data)
    print("更新成功")


if __name__ == "__main__":
    r = redis.Redis(connection_pool=pool)
    p = Pool(2)  
    while True:
        try:
            redis_val = r.rpop("keyword_word")
            if redis_val:
                redis_val = redis_val.decode()
                task = redis_val.split('^')
                # 如果搜索任务没有搜索标识，直接返回0
                p.apply_async(Dig_word, args=(int(task[0]), task[1],redis_val))
        except:
            pass

    p.close()  # 关闭进程池,不在接收新的任务
    p.join()  # 等待子进程全部运行完成，执行后续操作